package com.feheadline.ml.classification;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.util.Map;

import com.dtruth.dataflow.dao.ClassifierDAO;
import com.dtruth.dataflow.dao.HBaseDAO;
import com.dtruth.dataflow.util.StringUtil;

public class Sample2MySQL {

	public static void main(String[] args) {
		try {
			String tableName = "fe_class_sample_text";
			Text2MySQLIncre(tableName);
		} catch (IOException e) {
			e.printStackTrace();
		}

	}

	public static void HBase2MySQL(String tableName) throws IOException {
		long start = System.currentTimeMillis();
		HBaseDAO hbaseDao = new HBaseDAO();
		File path = new File("F:/machinelearning/classification");
		File[] files = path.listFiles();
		for (File f : files) {
			String aclass = f.getName();
			File[] sfiles = f.listFiles();
			if (sfiles == null || sfiles.length <= 0)
				continue;
			for (File sfile : sfiles) {
				String bclass = sfile.getName().substring(0,
						sfile.getName().indexOf(".txt"));
				FileReader fr = new FileReader(sfile);
				BufferedReader br = new BufferedReader(fr);
				String line = null;
				while ((line = br.readLine()) != null) {
					String key = StringUtil.getMD5(line);
					Map<String, Object> map = hbaseDao
							.selectRow("crawled", key);
					if (map.get("title") != null && map.get("article") != null)
						saveToMySQL(tableName, aclass.substring(2),
								bclass.substring(2), map.get("title")
										.toString(), map.get("article")
										.toString());
				}
				br.close();
				fr.close();
			}
		}
		long end = System.currentTimeMillis();
		System.out.println("time cost:" + (end - start) + "ms");
	}

	public static void Text2MySQL(String tableName) throws IOException {
		long start = System.currentTimeMillis();
		File path = new File("F:/machinelearning/classification/text");
		File[] files = path.listFiles();
		for (File f : files) {
			String aclass = f.getName();
			File[] sfiles = f.listFiles();
			if (sfiles == null || sfiles.length <= 0)
				continue;
			for (File sfile : sfiles) {
				InputStreamReader isr = new InputStreamReader(
						new FileInputStream(sfile), "gbk");
				BufferedReader br = new BufferedReader(isr);
				String line = null;
				StringBuilder title = new StringBuilder();
				StringBuilder article = new StringBuilder();
				int i = 0;
				while ((line = br.readLine()) != null) {
					if (i == 0) {
						title.append(line);
						i++;
					} else {
						if (StringUtil.containsCNStr(line))
							article.append(line);
					}

				}
				saveToMySQL(tableName, aclass, null, title.toString(),
						article.toString());
				br.close();
				isr.close();
			}
		}
		long end = System.currentTimeMillis();
		System.out.println("time cost:" + (end - start) + "ms");
	}

	public static void Text2MySQLIncre(String tableName) throws IOException {
		long start = System.currentTimeMillis();
		File path = new File("F:/machinelearning/样本库第二版20150313");
		File[] files = path.listFiles();
		for (File f : files) {
			String aclass = f.getName();
			File[] sfiles = f.listFiles();
			if (sfiles == null || sfiles.length <= 0)
				continue;
			for (File sfile : sfiles) {
				String bclass = sfile.getName();
				File[] txtfiles = sfile.listFiles();
				if (sfiles == null || sfiles.length <= 0)
					continue;
				for (File txtfile : txtfiles) {
					InputStreamReader isr = new InputStreamReader(
							new FileInputStream(txtfile), "unicode");
					BufferedReader br = new BufferedReader(isr);
					String line = null;
					String title = txtfile.getName().substring(0,
							txtfile.getName().indexOf(".txt"));
					StringBuilder article = new StringBuilder();
					while ((line = br.readLine()) != null) {
						if (line.trim().length() > 0)
							article.append(line);
					}
					if (StringUtil.containsCNStr(article.toString())) {
						// System.out.println(aclass + "\t" + bclass + "\t"
						// + title + "\t" + article.toString());
						// write2File(aclass + "\t" + bclass + "\t" + title +
						// "\t");
						if (aclass.equals(bclass))
							saveToMySQL(tableName, aclass, "", title,
									article.toString());
						else
							saveToMySQL(tableName, aclass, bclass, title,
									article.toString());
					}

					br.close();
					isr.close();
				}

			}
		}
		long end = System.currentTimeMillis();
		System.out.println("time cost:" + (end - start) + "ms");
	}

	public static void write2File(String str) {
		try {
			BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(
					new FileOutputStream("F:\\machinelearning\\output.txt",
							true)));
			bw.write(str + "\n");
			bw.close();
		} catch (IOException e) {
			e.printStackTrace();
		}

	}

	public static void Text2MySQLNews(String tableName) throws IOException {
		long start = System.currentTimeMillis();
		File f = new File("F:/machinelearning/classification/text/宏观");
		String aclass = f.getName();
		File[] sfiles = f.listFiles();
		if (sfiles == null || sfiles.length <= 0)
			System.out.println("wrong");
		for (File sfile : sfiles) {
			InputStreamReader isr = new InputStreamReader(new FileInputStream(
					sfile), "utf-8");
			BufferedReader br = new BufferedReader(isr);
			String title = sfile.getName().substring(0,
					sfile.getName().indexOf(".txt"));
			String line = null;
			StringBuilder article = new StringBuilder();
			while ((line = br.readLine()) != null) {
				if (StringUtil.containsCNStr(line))
					article.append(line);
			}
			saveToMySQL(tableName, aclass, "宏观政治", title, article.toString());
			br.close();
			isr.close();
		}
		long end = System.currentTimeMillis();
		System.out.println("time cost:" + (end - start) + "ms");
	}

	private static void saveToMySQL(String tableName, String aclass,
			String bclass, String title, String article) {
		ClassifierDAO cd = new ClassifierDAO();
		cd.save(tableName, aclass, bclass, title, article, "yjl");
		cd.close();
	}
}
